{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "default = pd.read_csv('../data/credit_card_default.csv')\n",
    "default.rename(columns=lambda x: x.lower(), inplace=True)\n",
    "default.rename(columns={'pay_0':'pay_1','default payment next month':'default'}, inplace=True)\n",
    "\n",
    "default['grad_school'] = (default['education'] == 1).astype(int)\n",
    "default['university'] = (default['education'] == 2).astype(int)\n",
    "default['high_school'] = (default['education'] == 3).astype(int)\n",
    "default.drop('education', axis=1, inplace=True)\n",
    "\n",
    "default['male'] = (default['sex']==1).astype(int)\n",
    "default['married'] = (default['marriage'] == 1).astype(int)\n",
    "default.drop(['sex','marriage'], axis=1, inplace=True)\n",
    "\n",
    "# For pay_n features if >0 then it means the customer was delayed on that month\n",
    "pay_features = ['pay_' + str(i) for i in range(1,7)]\n",
    "for p in pay_features:\n",
    "    default[p] = (default[p] > 0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy_features =['pay_'+str(i) for i in range(1,7)]\n",
    "dummy_features += ['male','married','grad_school','university','high_school']\n",
    "numerical_features = [x for x in default.columns if x not in dummy_features+['default']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_name = 'default'\n",
    "X = default.drop('default', axis=1)\n",
    "feature_names = X.columns\n",
    "robust_scaler = RobustScaler()\n",
    "X = robust_scaler.fit_transform(X)\n",
    "y = default[target_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "variances = pd.Series(default.var(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdsAAAF1CAYAAACgbECqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XucXWV97/HP1yg3g8hNa1SMIgblUmgHW3qiBOSoaLygtIjVAtamaHtsbe3RVkspeqrWHtsK9RIoggUsFgtatQ1WjRVUzAQhAQERiCIgCsj9opDf+WOv9GzHuezJ7JW9Z+bzfr32a9Ze61nP/q1nVuY7z9prslNVSJKk9jxi0AVIkjTXGbaSJLXMsJUkqWWGrSRJLTNsJUlqmWErSVLLDFtpCkl2S3JPkgWDrmWmkrwhyS3N8ezcQ/sNSQ7dErX1U5J3Jbk1yQ8GXYsEhq3mmCSrkpw4zvqXJflBkkdOt8+q+l5VLayqh/tT5WAkeRTwfuD5zfHcNuia2pDkycAfA8+qql+YYV/Lkny/P5VpPjNsNdecDrw2Scasfy1wVlU9NJ3ONiech9jjgW2AKwZdSMueAtxWVT8cdCFz7PzRDBi2mmvOB3YCnrNpRZIdgeXAx5rnL07yzSR3JbkhyQldbRcnqSS/neR7wBe71j2yaXNskiuT3J3kuiS/27X/siTfT/LHSX6Y5OYkx3Zt3zbJ/03y3SR3JrkwybbNtl9N8tUkdyS5LMmyrv2OaV7r7iTXJ/nN8Q4+ydZJ/i7JTc3j75p1zwCubprdkeSLE+z/2qa225K8fcy205O8a+yxdj3fkORPkqxLcm+Sf0zy+CT/3tT9n833onucj22+Bz9OclySA5r970hyctcx3Z5kn67XelyS+5PsOqbGQ4HPA4uaS+Wn9zC2434/kzwa+Peuvu5JsqjHcXhrknXAvUke2ez3ySQ/ar5/b+pq/+wko835eEuS94/3vdEsV1U+fMypB3AKcGrX898FLu16vgzYh84vm/sCtwAvb7YtBopOMD8a2LZr3SObNi8GdgcCHATcB/xSV98PAScCjwJe1Gzfsdn+D8Bq4InAAuDXgK2b57c17R8B/M/m+a5NHXcBS5o+ngDsNcGxnwh8HXhcs+9XgXeOObZHTrDvs4B7gOc2Nb2/OZZDm+2nA+8aM47f73q+oXntxzfH80PgEmD/pr8vAn8xppYP05ltPx94gM4vS4/r2v+gpv0Hgfd2vdYfAP82wXGMrWvCse3x+/n9Mf33Mg6XAk+mc/48AlgLHA9sBTwNuA54QdP+a8Brm+WFwK8O+t+Qj/4/nNlqLjoD+PVNM0bgt5p1AFTV6qpaX1Ubq2od8HE6P2S7nVBV91bV/WM7r6rPVtW11fFl4AK6ZtLAT4ETq+qnVfU5OgG2JMkjgNcBf1BVN1bVw1X11ap6EHgN8Lmq+lxT1+eBUToBAbAR2DvJtlV1c1VNdCn4N5vX/mFV/Qj4SzqX0HtxBPCZqvqvpqY/b153Ok6qqluq6kbgK8DFVfXNpr/z6ARvt3dW1QNVdQFwL/DxpvZN+29qfwbw6mYMaY7pn3qsadKx7eH7uTk+UFU3NOfPAXSC/cSq+klVXUfnF8JXNW1/Cjw9yS5VdU9VfX2Gr60hZNhqzqmqC4EfAS9L8jQ6P+zO3rQ9ya8k+VJzSe9O4DhglzHd3DBR/0kOS/L15tLmHXR+aHfvf1v97HvD99GZsexCZxZ37TjdPoXOLwh3bHoAS4EnVNW9wJFNnTcn+WySPScobxHw3a7n323W9WIRXcfdvO50b6K6pWv5/nGeL9yc9lV1MZ0wPqg59qcDn+6xpgnHFnr6fm6O7vPnKXQuRXe//p/RuQIA8NvAM4CrkqxJsnyGr60h5Jv3mqs+RmdGuwS4oKq6f4ifDZwMHFZVDyT5O37+h+u4H4eVZGvgk03fn6qqnyY5n84lyKncSudS6e7AZWO23QD8U1X9zng7VtUqYFUzW38XnZnReLOvm+j8cN80892tWdeLm4FnbnqSZDug+8+D7gW263o+ozt9N8MZdGapPwDOraoHetxvwrHt4fs53nnQyzh073cDcH1V7TFecVV1DXBUM2t/BXBukp2bX3Y0Rziz1Vz1MeBQ4HfouoTc2B64vQnaZwOvnka/W9F5//FHwENJDqPzfuOUqmojcBrw/uaGmQVJDmx+4J8JvCTJC5r12zQ33jypucnopc0NOw/SuSw90Z8hfRx4R5Jdk+xC533CM3s8tnOB5UmWJtmKzvu/3T8jLgVelGSnJL8A/GGP/fbLPwGH0wncj01jvwnHlqm/n7cAOyfZoWvddMfhG8BdzU1T2zY17J3kAIAkr0mya3N+3NHsM6v/zEw/z7DVnFRVG+jcHPRofv5y4xuBE5PcTSeMPjGNfu8G3tTs82M6Qd3r5UyAtwDrgTXA7cB7gUdU1Q3Ay+hcXvwRndnQn9D5N/oIOn83elOzz0HNMYznXXTej1zXvM4lzbpeju0K4PfozPxvbo6v+29M/4nOjHwDnfc1z+ml336pqu/TOZ6i835ur/tNOLZTfT+r6io6v8Bc11wCXsQ0x6E6f5/9EmA/4Ho6VzhOBTYF+AuBK5LcA/w98KppzNo1S6TKD4+XNDskOQ24qareMehapOnwPVtJs0KSxXTe0xx7R7M09LyMLGnoJXkncDnwvqq6ftD1SNPlZWRJklrmzFaSpJYZtpIktcwbpHq0yy671OLFiwddhuaSq5vPBViyZLB1SNpsa9euvbWqdp2qnWHbo8WLFzM6OjroMjSXLFvW+bp69SCrkDQDSb47dSsvI0uS1DrDVpKklhm2kiS1zLCVJKllhq0kSS3zbuQerb/xTha/7bODLkNzyD9f1/lc9ld5Xklb1Ib3vHiLv6YzW0mSWmbYSpLUMsNWkqSWDSxsk4wk+UDLr/HV5uviJK9u87UkSZrIwMK2qkar6k0z7SfJhDd5VdWvNYuLAcNWkjQQfQvbZvZ4edfztyQ5IcnqJO9N8o0k307ynGb7siSfSfKIJBuSPLZr3+8keXySXZN8Msma5vE/mu0nJFmZ5ALgY0n2avq/NMm6JHs07e5punwP8Jxm+5uTfCXJfl2vd1GSffs1FpIkddtSf/rzyKp6dpIXAX8BHLppQ1VtTPIp4HDgo0l+BdhQVbckORv426q6MMluwCrgmc2uvwwsrar7k5wE/H1VnZVkK2DBmNd/G/CWqloOkOR24BjgD5M8A9i6qtaNLTrJCmAFwILHTPmhDpIkjWtLXUb+1+brWjqXdMc6BziyWX5V8xw6oXxykkuBTwOPSbJ9s+3TVXV/s/w14M+SvBV4Stf6ifwLsDzJo4DXAaeP16iqVlbVSFWNLNhuhym6lCRpfP0M24fG9LdN1/KDzdeHGX82/TXg6Ul2BV7O/w/nRwAHVtV+zeOJVXV3s+3eTTtX1dnAS4H7gVVJDpms0Kq6D/g88DLgN4Czezg+SZI2Sz/D9hbgcUl2TrI1sLzXHauqgPOA9wNXVtVtzaYLgN/f1K77fdZuSZ4GXFdVH6AzAx77/uvdwPZj1p0KfABYU1W391qrJEnT1bewraqfAicCFwOfAa6aZhfnAK/h/19CBngTMNLc9PQt4LgJ9j0SuLy53Lwn8LEx29cBDyW5LMmbm3rXAncBH51mnZIkTUtfb5BqZpYT/u1sVd1K855tVa0GVndtGwUyTvsjGaOqThjz/N3Au8dpt7D5+lPged3bkiyi88vGBZMckiRJMzYv/wepJL9FZwb+9qraOOh6JElz27z81J+q+hg/f6lZkqRWpHNvkqYyMjJSo6Ojgy5Dc8myZZ2vq1cPsgpJM5BkbVWNTNVuXl5GliRpSzJsJUlqmWErSVLLDFtJklpm2EqS1DLDVpKklhm2kiS1zLCVJKllhq0kSS0zbCVJaplhK0lSywxbSZJaZthKktQyw1aSpJbNy8+z3RyX3X0fv/ClSwddhuaQf73jHgBe4XklbTE/OHi/gbyuM1tJklpm2EqS1DLDVpKkls3JsE3y1Wm2PyHJW9qqR5I0v83qsE3yyDHPFwBU1a8NpiJJkn7eQMI2yeIkVyU5NcnlSc5KcmiSi5Jck+TZzeOrSb7ZfF3S7HtMkn9J8m/ABUmWJflSkrOB9U2be7pe60+SrEmyLslfdq1/e5Krk/wnsGQLD4EkaR4Z5J/+PB34dWAFsAZ4NbAUeCnwZ8BvAc+tqoeSHAr8FfDKZt8DgX2r6vYky4BnA3tX1fXdL5Dk+cAezfYAn07yXOBe4FXA/nTG4BJg7dgCk6xo6uMRj39C3w5ckjS/DDJsr6+qTTPRK4AvVFUlWQ8sBnYAzkiyB1DAo7r2/XxV3d71/Btjg7bx/Obxzeb5Qjrhuz1wXlXd17z+p8crsKpWAisBHrXkWbVZRylJmvcG+Z7tg13LG7ueb6TzS8A7gS9V1d7AS4BtutrfO6avsc83CfDuqtqveTy9qv6x2WZ4SpK2iGG+QWoH4MZm+ZjN7GMV8LokCwGSPDHJ44D/Ag5Psm2S7emEuSRJrRjm/67xr+lcRv4j4Iub00FVXZDkmcDXkgDcA7ymqi5Jcg5wKfBd4Ct9qlmSpJ+TKq+m9uJRS55VO3/47EGXoTnkX9/8egBe8benDrgSaf7o9/+NnGRtVY1M1W6YLyNLkjQnGLaSJLVsmN+zHSq/uP12jA7oo5k0Rz12ITC4j/yStOU4s5UkqWWGrSRJLTNsJUlqmWErSVLLDFtJklpm2EqS1DLDVpKklhm2kiS1zLCVJKllhq0kSS0zbCVJaplhK0lSywxbSZJa5qf+9Ojuu9fzhS/uPugyNIf80h03AXCJ55W0WZ53yLWDLqFnzmwlSWqZYStJUssMW0mSWjZvwjbJsiSfGXQdkqT5Z96ErSRJgzKrwjbJ4iRXJTk1yeVJzkpyaJKLklyT5NnN46tJvtl8XTJOP49OclqSNU27lw3ieCRJ88OsCtvG04G/B/YF9gReDSwF3gL8GXAV8Nyq2h84Hvircfp4O/DFqjoAOBh4X5JHb4HaJUnz0Gz8O9vrq2o9QJIrgC9UVSVZDywGdgDOSLIHUMCjxunj+cBLk7yleb4NsBtwZXejJCuAFQCPe9xsHCpJ0jCYjQnyYNfyxq7nG+kczzuBL1XV4UkWA6vH6SPAK6vq6sleqKpWAisBlizZumZUtSRp3pqNl5GnsgNwY7N8zARtVgH/K0kAkuy/BeqSJM1TczFs/xp4d5KLgAUTtHknncvL65Jc3jyXJKkVs+oyclVtAPbuen7MBNue0bXbnzfbV9NcUq6q+4HfbbFUSZL+21yc2UqSNFQMW0mSWjarLiMP0vbb78PzDhkddBmaSx67DIDnHbJ6oGVIap8zW0mSWmbYSpLUMsNWkqSWGbaSJLXMsJUkqWWGrSRJLTNsJUlqmWErSVLLDFtJklpm2EqS1DLDVpKklhm2kiS1zLCVJKllhq0kSS3zI/Z6dNNNN3HCCScMugzNIcds2ADA6Z5Xmofm289TZ7aSJLXMsJUkqWWGrSRJLTNsJUlq2RYP2yQbkuzSh36OSXLyMNUkSdJ4+hK2SbyrWZKkCfQUtkn+PMlVST6f5ONJ3pJkdZK/SvJl4A+SvCTJxUm+meQ/kzy+2XfnJBc06z8CZJLXeXSSzya5LMnlSY5s1h+Q5KvN+m8k2b7ZZVGS/0hyTZK/7urnqCTrmz7eO9X6SepZkWQ0yeh9993Xy1BJkvRzppyRJhkBXgns37S/BFjbbH5sVR3UtNsR+NWqqiSvB/438MfAXwAXVtWJSV4MrJjk5V4I3FRVL2763CHJVsA5wJFVtSbJY4D7m/b7NXU9CFyd5CTgYeC9wC8DPwYuSPJy4Bvjra+q8ycqpqpWAisBFi1aVFONlSRJ4+nl8u9S4FNVdT9Akn/r2nZO1/KTgHOSPAHYCri+Wf9c4BUAVfXZJD+e5LXWA3/TzDo/U1VfSbIPcHNVrWn6uKupA+ALVXVn8/xbwFOAnYHVVfWjZv1ZTQ01wfoJw1aSpH7o5TLyhJd9gXu7lk8CTq6qfYDfBbbp2tbTrLCqvk1n5rkeeHeS45vXn2j/B7uWH6bzy8NE9U52HJIktaaXsL0QeEmSbZIsBF48QbsdgBub5aO71v8X8JsASQ4DdpzohZIsAu6rqjOBvwF+CbiKznuzBzRttp/ihqyLgYOS7JJkAXAU8OVJ1kuS1KopLyM375N+GrgM+C4wCtw5TtMTgH9JciPwdeCpzfq/BD6e5BI64fa9SV5uH+B9STYCPwXeUFU/aW6UOinJtnTerz10knpvTvKnwJfozGY/V1WfAphovSRJbUrV1Fd4kyysqnuSbEdnprqiqi5pvbohsmjRolqxYrJ7u6TpOeb00wE4/ZhjBlqHNAhz5YMIkqytqpGp2vX697ErkzyLzvuwZ8y3oJUkaSZ6mtn2/UWTnYEvjLPpeVV125aupxcjIyM1Ojo66DI0lyxb1vm6evUgq5A0A/2e2fZVE6j7DeK1JUna0vwgAkmSWmbYSpLUMsNWkqSWGbaSJLXMsJUkqWWGrSRJLTNsJUlqmWErSVLLDFtJklpm2EqS1DLDVpKklhm2kiS1zLCVJKllA/nUn9noJzfew/ff9pVBl6E5ZNfr7gTgR55Xs9qT3vOcQZegWcCZrSRJLTNsJUlqmWErSVLLDFtJklo2L8M2ye8n+U6SSrLLoOuRJM1t8zJsgYuAQ4HvDroQSdLcN7Rhm2RxkquSnJFkXZJzk2yX5Pgka5JcnmRlOnZPcknXvnskWTtR31X1zara0EMNK5KMJhm9/b47+nRkkqT5ZmjDtrEEWFlV+wJ3AW8ETq6qA6pqb2BbYHlVXQvcmWS/Zr9jgdNn+uJVtbKqRqpqZKftHjvT7iRJ89Swh+0NVXVRs3wmsBQ4OMnFSdYDhwB7NdtPBY5NsgA4Ejh7i1crSdI4hj1sa5znHwSOqKp9gFOAbZptnwQOA5YDa6vqti1WpSRJkxj2sN0tyYHN8lHAhc3yrUkWAkdsalhVDwCrgA8BH92iVUqSNIlhD9srgaOTrAN2ohOkpwDrgfOBNWPan0Vn9nvBZJ0meVOS7wNPAtYlObXfhUuStMmwfxDBxqo6bsy6dzSP8SwFTquqhyfrtKo+AHygD/VJkjSlYQ/bniU5D9idzk1TfbfVExf66R7qr6/vAPipMdJ8MLRh2/wd7N7TaH/42HVNAD91zOq3VtWqmVUnSVLvhjZs+2G8AJYkaUsb9hukJEma9QxbSZJaZthKktQyw1aSpJYZtpIktcywlSSpZYatJEktM2wlSWqZYStJUssMW0mSWmbYSpLUMsNWkqSWzekPIuinW677Dv/3yOWDLkNzyG98az0An5ij59Ufn/OZQZcgDQ1ntpIktcywlSSpZYatJEktM2wlSWpZT2GbZHGSy8dZf2KSQ6fY94Qkb9ncAifp95gkJ/eprw1JdulHX5IkjTWju5Gr6vh+FSJJ0lw1ncvIC5KckuSKJBck2TbJ6UmOAEjyoiRXJbkwyQeSdN/3/6wkq5Ncl+RNE71Akkcn+WySy5JcnuTIZv0BSb7arP9Gku2bXRYl+Y8k1yT5665+jkqyvunjvVOtlySpTdOZ2e4BHFVVv5PkE8ArN21Isg3wEeC5VXV9ko+P2XdP4GBge+DqJB+qqp+O8xovBG6qqhc3/e6QZCvgHODIqlqT5DHA/U37/YD9gQebfk8CHgbeC/wy8GPggiQvB74x3vqqOn+iA06yAlgBsON22/Y2SpIkjTGdme31VXVps7wWWNy1bU/guqq6vnk+Nmw/W1UPVtWtwA+Bx0/wGuuBQ5O8N8lzqupOYAlwc1WtAaiqu6rqoab9F6rqzqp6APgW8BTgAGB1Vf2oaXcW8NxJ1k+oqlZW1UhVjTx6660maypJ0oSmE7YPdi0/zM/OijODff9bVX2bzsxzPfDuJMc3fdc0+p2olqlqlCSpFf3605+rgKclWdw8P3JzOkmyCLivqs4E/gb4pabvRUkOaNpsn2Syy98XAwcl2SXJAuAo4MuTrJckqVV9+b+Rq+r+JG8E/iPJrXTeH90c+wDvS7IR+Cnwhqr6SXOj1ElJtqXzfu2Ef25UVTcn+VPgS3Rms5+rqk8BTLRekqQ2pWqiK7TT7ChZWFX3JAnwD8A1VfW3fel8CDx5p8fWH/7PpYMuQ3PIb3zpawB84uADB1xJO/wgAs0HSdZW1chU7fr5P0j9TpJLgSuAHejcnSxJ0rzXt5nttF402Rn4wjibnldVt23penoxMjJSo6Ojgy5Dc8myZZ2vq1cPsgpJM9DrzHYgn2fbBOp+g3htSZK2ND+IQJKklhm2kiS1zLCVJKllhq0kSS0zbCVJaplhK0lSywxbSZJaZthKktQyw1aSpJYZtpIktcywlSSpZYatJEktM2wlSWrZQD71Zzb64Xfv5h+O++Kgy9Ac8vJv3wHA+UN6Xv3ehw8ZdAnSnOHMVpKklhm2kiS1zLCVJKllhq0kSS2bl2Gb5KwkVye5PMlpSR416JokSXPXvAxb4CxgT2AfYFvg9YMtR5I0lw1t2CZZnOSqJGckWZfk3CTbJTk+yZpmVroyHbsnuaRr3z2SrJ2o76r6XDWAbwBPmqCGFUlGk4ze88Ad/T9ISdK8MLRh21gCrKyqfYG7gDcCJ1fVAVW1N51Z6fKquha4M8l+zX7HAqdP1Xlz+fi1wH+Mt72qVlbVSFWNLNzmsTM/GknSvDTsYXtDVV3ULJ8JLAUOTnJxkvXAIcBezfZTgWOTLACOBM7uof8PAv9VVV/pc92SJP23YQ/bGuf5B4Ejqmof4BRgm2bbJ4HDgOXA2qq6bbKOk/wFsCvwR32tWJKkMYY9bHdLcmCzfBRwYbN8a5KFwBGbGlbVA8Aq4EPARyfrNMnrgRcAR1XVxr5XLUlSl2EP2yuBo5OsA3aiE6SnAOuB84E1Y9qfRWf2e8EU/X4YeDzwtSSXJjm+r1VLktRl2D+IYGNVHTdm3Tuax3iWAqdV1cOTdVpVw37ckqQ5ZM6ETpLzgN3p3DQlSdLQSOdPTeemJoCfOmb1W6tq1XT7GhkZqdHR0f4UJgEsW9b5unr1IKuQNANJ1lbVyFTt5szMdjxVdfiga5AkadhvkJIkadYzbCVJaplhK0lSywxbSZJaZthKktQyw1aSpJYZtpIktcywlSSpZYatJEktM2wlSWqZYStJUssMW0mSWmbYSpLUsjn9qT/99MDlV3Dlns8cdBmaQ3b73ncB+N6AzqtnXnXlQF5Xmo+c2UqS1DLDVpKklhm2kiS1bN6GbZJ/THJZknVJzk2ycNA1SZLmpnkbtsCbq+oXq2pf4HvA7w+6IEnS3DTUYZtkcZKrkpzRNQPdLsnxSdYkuTzJynTsnuSSrn33SLJ2or6r6q6mXYBtgWr/iCRJ89FQh21jCbCymYHeBbwROLmqDqiqvekE5fKquha4M8l+zX7HAqdP1nGSjwI/APYEThpn+4oko0lGb3/4ob4dkCRpfpkNYXtDVV3ULJ8JLAUOTnJxkvXAIcBezfZTgWOTLACOBM6erOOqOhZYBFzZtB+7fWVVjVTVyE4L/JNkSdLmmQ1hO/bybgEfBI6oqn2AU4Btmm2fBA4DlgNrq+q2KTuvehg4B3hl3yqWJKnLbAjb3ZIc2CwfBVzYLN/a3EF8xKaGVfUAsAr4EPDRiTps3uN9+qZl4CXAVS3ULknSrPjvGq8Ejk7yEeAaOkG6I7Ae2ACsGdP+LOAVwAWT9BngjCSPaZYvA97Q37IlSeqYDWG7saqOG7PuHc1jPEuB05rLw+Oqqo3A/+hTfZIkTWo2hG3PkpwH7E7npilJkoZCqub2n5c2AfzUMavfWlWrptPPyMhIjY6O9q8wadmyztfVqwdZhaQZSLK2qkamajenZrbjqarDB12DJGl+mw13I0uSNKsZtpIktcywlSSpZYatJEktM2wlSWqZYStJUssMW0mSWmbYSpLUMsNWkqSWGbaSJLXMsJUkqWWGrSRJLTNsJUlq2Zz/1J9+ueK2K9jnjH0GXYbmkNN+cB0ArxtzXq0/ev0gypHUIme2kiS1zLCVJKllhq0kSS0zbCVJatm8D9skJyW5Z9B1SJLmrnkdtklGgMcOug5J0tw21GGbZHGSq5KckWRdknOTbJfk+CRrklyeZGU6dk9ySde+eyRZO0nfC4D3Af97SxyLJGn+GuqwbSwBVlbVvsBdwBuBk6vqgKraG9gWWF5V1wJ3Jtmv2e9Y4PRJ+v194NNVdfNEDZKsSDKaZPThux/ux7FIkuah2RC2N1TVRc3ymcBS4OAkFydZDxwC7NVsPxU4tpm1HgmcPV6HSRYBvw6cNNkLV9XKqhqpqpEF2y/ow6FIkuaj2RC2Nc7zDwJHVNU+wCnANs22TwKHAcuBtVV12wR97g88HfhOkg3Adkm+0+/CJUmC2RG2uyU5sFk+CriwWb41yULgiE0Nq+oBYBXwIeCjE3VYVZ+tql+oqsVVtRi4r6qe3kr1kqR5bzaE7ZXA0UnWATvRCdJTgPXA+cCaMe3PojP7vWBLFilJ0kRmwwcRbKyq48ase0fzGM9S4LSq6vmOpqpauLnFSZI0ldkQtj1Lch6wO52bpiRJGgqpGnv/0dzSBPBTx6x+a1Wtmk4/IyMjNTo62r/CpGXLOl9Xrx5kFZJmIMnaqhqZqt2cmtmOp6oOH3QNkqT5bTbcICVJ0qxm2EqS1DLDVpKklhm2kiS1zLCVJKllhq0kSS0zbCVJaplhK0lSywxbSZJaZthKktQyw1aSpJYZtpIktcywlSSpZXP+U3/65qZvwgk7DLoKDbsT7hx0BZKGkDNbSZJaZthKktQyw1aSpJYZtpIktWxehm06/k+Sbye5MsmbBl2TJGnumq93Ix8DPBnYs6o2JnncgOuRJM1hQzuzTbI4yVVJzkiyLsm5SbZLcnySNUkuT7KymaXunuSSrn33SLJ2ku7fAJxYVRsBquqHE9SwIsloktEf3Vf9PUBJ0rwxtGHbWAKsrKp9gbuANwInV9UBVbU3sC2wvKquBe5Msl+z37HA6ZP0uztwZBOk/55kj/EaVdXKqhqpqpFdt0u/jkmSNM8Me9jeUFVWQtTnAAAHVElEQVQXNctnAkuBg5NcnGQ9cAiwV7P9VODYJAuAI4GzJ+l3a+CBqhoBTgFOa6V6SZIY/rAde+22gA8CR1TVPnSCcptm2yeBw4DlwNqqum2Sfr/ftAc4D9i3bxVLkjTGsIftbkkObJaPAi5slm9NshA4YlPDqnoAWAV8CPjoFP2eT2dWDHAQ8O2+VSxJ0hjDfjfylcDRST4CXEMnSHcE1gMbgDVj2p8FvAK4YIp+3wOcleTNwD3A6/tYsyRJP2PYw3ZjVR03Zt07msd4lgKnVdXDk3VaVXcAL+5DfZIkTWnYw7ZnSc6jc5fxIVO1lSRpSxrasK2qDcDe02h/+Nh1TQA/dczqt1bVqmkXtGh/OGF02rtJkjS0YdsP4wWwJElb2rDfjSxJ0qxn2EqS1DLDVpKklhm2kiS1zLCVJKllhq0kSS0zbCVJaplhK0lSywxbSZJaZthKktQyw1aSpJYZtpIktcywlSSpZXP6U3/6af2Nd7L4bZ8ddBkaIhve8+JBlyBplnBmK0lSywxbSZJaZthKktSyeRm2SU5Pcn2SS5vHfoOuSZI0d83nG6T+pKrOHXQRkqS5b2hntkkWJ7kqyRlJ1iU5N8l2SY5PsibJ5UlWpmP3JJd07btHkrWDrF+SpE2GNmwbS4CVVbUvcBfwRuDkqjqgqvYGtgWWV9W1wJ1dl4OPBU6fou//04T43ybZerwGSVYkGU0y+vB9d/blgCRJ88+wh+0NVXVRs3wmsBQ4OMnFSdYDhwB7NdtPBY5NsgA4Ejh7kn7/FNgTOADYCXjreI2qamVVjVTVyILtdpj50UiS5qVhD9sa5/kHgSOqah/gFGCbZtsngcOA5cDaqrptwk6rbq6OB4GPAs/ue+WSJDWGPWx3S3Jgs3wUcGGzfGuShcARmxpW1QPAKuBDdAJ0Qkme0HwN8HLg8j7XLUnSfxv2u5GvBI5O8hHgGjpBuiOwHtgArBnT/izgFcAFU/R7VpJdgQCXAsf1sWZJkn7GsIftxqoaG4TvaB7jWQqcVlUPT9ZpVR3Sj+IkSerFsIdtz5KcB+xO56YpSZKGxtCGbVVtAPaeRvvDx65rAvipY1a/tapWTbeefZ64A6N+yoskaTMMbdj2w3gBLEnSljbsdyNLkjTrGbaSJLXMsJUkqWWGrSRJLTNsJUlqmWErSVLLUjX2//rXeJLcDVw96DpmiV2AWwddxCzhWPXGceqdY9W7fozVU6pq16kazem/s+2zq6tqZNBFzAZJRh2r3jhWvXGceudY9W5LjpWXkSVJaplhK0lSywzb3q0cdAGziGPVO8eqN45T7xyr3m2xsfIGKUmSWubMVpKklhm2QJIXJrk6yXeSvG2c7VsnOafZfnGSxV3b/rRZf3WSF2zJure0zR2nJIuT3J/k0ubx4S1d+5bWw1g9N8klSR5KcsSYbUcnuaZ5HL3lqh6MGY7Vw13n1ae3XNWD0cNY/VGSbyVZl+QLSZ7StW3enFczHKd2zqmqmtcPYAFwLfA0YCvgMuBZY9q8Efhws/wq4Jxm+VlN+63pfG7utcCCQR/TEI7TYuDyQR/DkI3VYmBf4GPAEV3rdwKua77u2CzvOOhjGsaxarbdM+hjGLKxOhjYrll+Q9e/wXlzXs1knNo8p5zZwrOB71TVdVX1E+CfgZeNafMy4Ixm+VzgeUnSrP/nqnqwqq4HvtP0NxfNZJzmmynHqqo2VNU6YOOYfV8AfL6qbq+qHwOfB164JYoekJmM1XzTy1h9qarua55+HXhSszyfzquZjFNrDFt4InBD1/PvN+vGbVNVDwF3Ajv3uO9cMZNxAnhqkm8m+XKS57Rd7IDN5LyYT+cUzPx4t0kymuTrSV7e39KGznTH6reBf9/MfWezmYwTtHRO+T9IwXgzr7G3aE/Uppd954qZjNPNwG5VdVuSXwbOT7JXVd3V7yKHxEzOi/l0TsHMj3e3qropydOALyZZX1XX9qm2YdPzWCV5DTACHDTdfeeAmYwTtHROObPt/Nbz5K7nTwJumqhNkkcCOwC397jvXLHZ49RcZr8NoKrW0nk/5RmtVzw4Mzkv5tM5BTM83qq6qfl6HbAa2L+fxQ2ZnsYqyaHA24GXVtWD09l3jpjJOLV3Tg36zexBP+jM7q+jc4PTpjfT9xrT5vf42Rt/PtEs78XP3iB1HXP3BqmZjNOum8aFzk0LNwI7DfqYBjlWXW1P5+dvkLqezk0sOzbLjtX4Y7UjsHWzvAtwDWNuhJlLjx7/De5P55fZPcasnzfn1QzHqbVzauADMwwP4EXAt5vBf3uz7kQ6v/EAbAP8C50boL4BPK1r37c3+10NHDboYxnGcQJeCVzRnPSXAC8Z9LEMwVgdQOc38HuB24AruvZ9XTOG3wGOHfSxDOtYAb8GrG/Oq/XAbw/6WIZgrP4TuAW4tHl8ej6eV5s7Tm2eU/4PUpIktcz3bCVJaplhK0lSywxbSZJaZthKktQyw1aSpJYZtpIktcywlSSpZYatJEkt+3+ATbwo/WcXbwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 504x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(7,6))\n",
    "variances.loc[dummy_features].sort_values().plot(kind='barh', ax=ax)\n",
    "ax.vlines(0.1, ymin=-1, ymax=25, colors='red')\n",
    "ax.set_title('Variances of dummy features');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SelectKBest(k='all', score_func=<function chi2 at 0x000001E68012A8C8>)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_selector = SelectKBest(chi2, k=\"all\")\n",
    "dummy_selector.fit(default[dummy_features], default[target_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3141.39566361, 2920.6909149 , 2222.60680108, 2010.51973814,\n",
       "       1926.49180212, 1630.3290955 ,   28.92210682,   14.48674466,\n",
       "         51.149551  ,   21.23797331,   26.33631512])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_selector.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 7.53480421e-08, 1.41149283e-04,\n",
       "       8.55902557e-13, 4.05647636e-06, 2.86844139e-07])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_selector.pvalues_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import f_classif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SelectKBest(k='all', score_func=<function f_classif at 0x000001E68012A7B8>)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_selector = SelectKBest(f_classif, k=\"all\")\n",
    "num_selector.fit(default[numerical_features], default[target_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.30224395e-157, 1.61368459e-002, 6.67329549e-004, 1.39573624e-002,\n",
       "       1.47699827e-002, 7.85556416e-002, 2.41634443e-001, 3.52122521e-001,\n",
       "       1.14648761e-036, 3.16665676e-024, 1.84177029e-022, 6.83094160e-023,\n",
       "       1.24134477e-021, 3.03358907e-020])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_selector.pvalues_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5    bill_amt4\n",
       "6    bill_amt5\n",
       "7    bill_amt6\n",
       "dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(numerical_features).loc[num_selector.pvalues_>0.05]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features: 25\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of features:\", X.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import RFE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "RF = RandomForestClassifier(n_estimators=100, max_depth=20, max_features='auto',\n",
    "                            random_state=55, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "recursive_selector = RFE(estimator=RF, n_features_to_select=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "recursive_selector = recursive_selector.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True, False, False, False, False, False,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True, False,\n",
       "       False, False, False, False, False, False, False])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recursive_selector.support_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 most important features:\n",
      "limit_bal\n",
      "age\n",
      "pay_1\n",
      "bill_amt1\n",
      "bill_amt2\n",
      "bill_amt3\n",
      "bill_amt4\n",
      "bill_amt5\n",
      "bill_amt6\n",
      "pay_amt1\n",
      "pay_amt2\n",
      "pay_amt3\n"
     ]
    }
   ],
   "source": [
    "print('12 most important features:')\n",
    "for x in feature_names[recursive_selector.support_]:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features to eliminate:\n",
      "pay_2\n",
      "pay_3\n",
      "pay_4\n",
      "pay_5\n",
      "pay_6\n",
      "pay_amt4\n",
      "pay_amt5\n",
      "pay_amt6\n",
      "grad_school\n",
      "university\n",
      "high_school\n",
      "male\n",
      "married\n"
     ]
    }
   ],
   "source": [
    "print('Features to eliminate:')\n",
    "for x in feature_names[~recursive_selector.support_]:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "selector_model = cross_validate(estimator=RF,X=X[:,recursive_selector.support_], y=y,\n",
    "                            scoring=['recall','accuracy'], cv=10, n_jobs=-1)\n",
    "\n",
    "full_model = cross_validate(estimator=RF,X=X,y=y,\n",
    "                            scoring=['recall','accuracy'], cv=10, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = pd.DataFrame({'Full Model Recall':full_model['test_recall'],\n",
    "                        'Full Model Accuracy':full_model['test_accuracy'],\n",
    "                        'Selector Model Recall': selector_model['test_recall'],\n",
    "                        'Selector Model Accuracy': selector_model['test_accuracy']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Full Model Recall        0.361365\n",
       "Selector Model Recall    0.355791\n",
       "dtype: float64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics[['Full Model Recall','Selector Model Recall']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAE/CAYAAACq86yCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGY1JREFUeJzt3XuUpVV95vHvY7dyEWhpaA0i0JogioggHeMVJN7Au4LxDmTMOJkw41Izs8aICxGjojErMaOOoKOi4wTUKCEqDqjgDUG5dkelCXKRiwqIXCUI+Js/3l1yqK7uquoqqqprfz9rndXnvO9+97t/5y3OeWq/u5tUFZIkST2633wPQJIkab4YhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJElzJMnPkzy1PT8mycc20PaNSa5NcmuSB87dKOdHkgOSXDLFtht876TpMAhJHUlyeZLfJNl+3PYLklSSlXM4lle3L/lbk9ye5Lcjr2+dQb+PSnLXJG2OSXJnO9eNSb6TZNXGnnO2teDzXuBpVbVVVd02C33+PMmvW80/S/KxJFuM7D8hyR2j1yDJiyfoZ/P2s3JlkvuNbN8sya+S/PtMxyrNJYOQ1J/LgFeOvUjyWGCL9Te/b1TVZ9qX/FbAgcA1Y6/btvva8e08K4CzgBPn4JxTtQOwpKrWTvfAJPcbDSjjPLvVvAp4MvDfxu1/5+g1qKqTNnCqXwPPGHn9IuDa6Y5Xmm8GIak/nwYOGXl9KPCp0Qbtt/v3J/lpkl8k+cjY7EGSbZN8Kcl1bQbgS0keNnLsGUnemeS7SW5Jcur4GaipSrJTkn9Ocn2SS5P8+ci+pyQ5P8nNbbbjPW3Xt4AlI7Mae2/oHFV1J/B/gUck2Xqk/5ckWd1mjL6dZPeRfStHxnV9kr9t2x/V6r+hvT/Hj/Y5xZofC1w4UsMpbft+Sc5LclOSs5L84cgxZyU5OsnZDAHloZPUfDXwNWCv6YxtnPE/R4ew7s/Rzkm+0t6Pi5McOrLvgUk+097fNcDe445d77Uf1+6BbTbrhtbX2Um2nUFd6oxBSOrPWcA2SR6dZAnwcuD/jGvzXuCRDF+UfwDsCBzZ9t0P+ASwC7AzcDvwwXHHvwr4U+DBwANYd+ZhUm1sXwHOZPhiPwB4a5L9WpMPAu+uqm2AXYGx2Yt9gbtHZjXOn+Q8mwGvBX4O3Nq2PRH4cKthO4Yv/ZOSLE1yf+AU4Met/p2Afxrp8mjg94DHArsBR0yn7qpaA+wzUsOBSR4M/AtwTBvPR4CvJFk2cuhrGMLI1q2WDdW8M/BsYEprctbj88BzkmzVxrcPw/Ua9TlgLcMM16uAv0vylLbvXQzv00rghcBhI+Ob7NqP+jNgKcPP6PbAfwF+M4O61BmDkNSnsd/mnwVcBFw9tiNJgP8IvKmqbqiqW4B3A68AqKpfVtU/VdWv2753AeO/oD5RVRdX1e3AZ9m4mYenAptX1Xur6jdVdTFDAHtF238n8Mgk21XVLVV19jT7f22SGxlmUF4NHFz3/M8X/xPwwao6t6rurqrjgM0YvuyfCmwDvLW9B7dX1ZkAVXVRVX2jjffnwN+z7nuzMV4EXFBVn62qu6rqk8BVDLcUx3ysqtZW1Z1Vtb41UqckuQW4Argc+Otx+49osyo3JrlqkjHdCpwKHMQQcj7PcE0ASLIr8DiG9+mOqjoHOJ4hdAL8CcOtuBur6jLgQyN9T3btR93JcHvz99t784PZWFOlfhiEpD59muHL6zDG3c5g+FLZEjh37EsR+GrbTpItkxyb5IokNzPcinpQ+y1+zOiMxK+BjVnzswuwcuSL+UbgzQyzCDDc0tsTuLjdDnnONPv/dFU9iGG24ifcO6ztwjADMXruFQyzDjsBl1XVb8d3mOShST6X5Or23nyMYZZiph7KEF5GXdHGM+bKKfRzYFVtzTAbtAewfNz+d1XVg9rjYesevo5PMQTqdW6LtTFf18LwvcbcwvZDxo15tL7Jrv2o/w18E/h8kquSvHvcz6K0QQYhqUNVdQXDounnAl8Yt/t6httdjxn5Ulw2soD5Lxlu+fxRuy21b9ueWR7mlcBFI2N4UFVtXVUvaTX8uKpeznD77R+ALyR5AFAb6HMdVXUtwwzQe0bWMl0JHDnu3FtW1RfavpWZeEHy3wC3AXu09+bPmJ335RqGcDBqZ0Zm8phG3VV1GsPi8PfOcFxfY7iFukVV/WDcvmuAFRn5m2m0MbeZt2sZQuXovjEbvPbjarmjqo6sqkcx/Cy+jIlnjqQJGYSkfr0O+OPxtxHaTMdHGdZzPBggyY4jMy5bMwSlG5MsB95+H43vO+3cb8zwV7aXJtkzyePb9kPabbG7gZsYgsBvGb5gl7R1MFNSVasZZrb+sm06DvivSVZlsFWSFybZso3rFuCdbXZsiyRPbsdtzXDL6OZ2/jfP9E1oTgb2TnJwex8OYQgOX51Bn38LvCjJoze2g/az8lzgpRPsvgRYDfx1hsX3j2eYxftM2/9Zhltxy5LsAvzFyLEbvPajkjwzye4tmN4M3AXcvbE1qT8GIalTVfWTtm5jIv+D4YvsrHaL52sMs0AwrHvZgmHm6Cxm9mW8ofHdyfAl+2SG2ybXAf+Le26zPR9Y29a8vAf4k7ZG5FfA+7jn1t5U1yf9DXB4kuVV9V3gDcCxwI3AxQy3EmtkXI9jWKfzU+4JAkcyrG+5Cfgi915EvdGq6hcMC4qPAH7JsCD4+VV14wz6vAY4AXjbDMe2pqp+PMH2YlgHtDvDrdITgf9eVd9uTd7G8DP0U+DLjNxam8K1H7Uj8M8M4fRfGRZZf3YmNakvuWdtoCRJUl+cEZIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1K2l8z0AzY3tt9++Vq5cOd/DkCRpTpx77rnXV9WKydoZhDqxcuVKzjlnff9kjCRJi0uS8f9bmgl5a0ySJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6tbS+R6A5sg158NRy+Z7FJIkreuom+bt1M4ISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnq1rwGoSR3J7lg5LFykvaXJ9m+Pb91PW0qyadHXi9Ncl2SL01zbL8713TbtO1rkqxO8s0ku0zn3FMY29PH6klyWJIPzmb/kiT1Yr5nhG6vqr1GHpfPQp+3AXsk2aK9fhZw9Sz0O137V9WewBnA2+bh/JIkaRLzHYTWMX6GI8mXkjx9mt2cAjyvPX8l8I8j/S1PclKbrTkryZ5t+3ZJTk1yfpJjgYwc85ok32+zVscmWTKNsXwP2HGyvpIckOS8JBcm+Xrb9oQkZ7YxnZlkt2m+D5IkaQPmOwhtMXJb7Iuz2O8JwCuSbA7sCZw9su8dwPlttuatwKfa9rcD36mqvYGTgZ0BkjwaeDnwlKraC7gbePU0xnIAcNKG+kqyAvgocFBVPQ54WTv2ImDfNqYjgXdP47ySJGkSS+f5/Le3QDCrqmp1W2/0SuAr43Y/FTiotftGmwlaBuwLvLRt/3KSX7X2zwD2AX6QBGAL4NopDOP0JA9pbcduja2vrycC36qqy9r5b2jtlwHHJ9kVKOD+U3wLAEjyeuD1ADsvyyStJUnqz3zPCE3kLu49rs03sp+TgfczclusmSgR1Lg/x7c/fmQd025VddQUzr8/sAvwQ+DoSfrKes79TuD0qtoDeAHTfC+q6riqWlVVq1ZsaRCSJGm8hRiELgf2SnK/JDsBT9jIfj4OHF1Va8Zt/xbt1lZbe3R9Vd08bvuBwLat/deBg5M8uO1bPtW/BVZVtwNvBA5JsnwDfX0P2C/Jw8e2ty6Wcc9C78OmXrokSZqKhRiEvgtcBqxhmNE5b2M6qaqrquoDE+w6CliVZDVwDHBo2/4OYN8k5wHPBn7a+vkRw62tU9sxpwE7TGMcP2OYlTp8fX1V1XUMt7C+kORC4MR2+PuA9yT5LjCdBdqSJGkKUjXRHRktNqseuqTOef1W8z0MSZLWddRNs95lknOratVk7RbijJAkSdKcMAhJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktStpfM9AM2Rh+4NR50z36OQJGlBcUZIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3Vo63wPQ3Fhz9U2sfMuX53sYkqROXX7M8+Z7CBNyRkiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcmDUJJjkjywySrk1yQ5I8maf/JJAdPdyBJViZ51XSPm6CfSvLpkddLk1yX5EvT7OfyJNtvTJu2fU17z76ZZJfpnHsKY3v6WD1JDkvywdnsX5KkXmwwCCV5EvB84PFVtSfwTODK+2gsK4FpBaEkSyfYfBuwR5It2utnAVfPbGgbZf/2np0BvG0ezi9JkiYx2YzQDsD1VXUHQFVdX1XXACTZp812nJvk/yXZYfzB62uT5A+SfC3JhUnOS/L7wDHA09qs05uSbJ7kE21m5fwk+7djD0vyuST/Apy6nnGfAjyvPX8l8I8jY1qe5KQ2W3NWkj3b9u2SnNrOdSyQkWNek+T7bWzHJlky2Rs74nvAjpP1leSA9l5cmOTrbdsTkpzZxnRmkt2mcV5JkjSJyYLQqcBOSS5O8uEk+wEkuT/wP4GDq2of4OPAu0YPnKTNZ4APVdXjgCcDPwPeAny7qvaqqr8DDgeoqscyhJnjk2zejn8ScGhV/fF6xn0C8IrWfk/g7JF97wDOb7M1bwU+1ba/HfhOVe0NnAzs3Op4NPBy4ClVtRdwN/DqSd63UQcAJ22oryQrgI8CB7X35GXt2IuAfduYjgTePY3zSpKkSUx0a+l3qurWJPsATwP2B05M8hbgHGAP4LQkAEsYwsyo3SZqk2RrYMeq+mI7x78DtDajnsoQpKiqi5JcATyy7Tutqm7YwLhXJ1nJEKC+MkG/B7V232gzQcuAfYGXtu1fTvKr1v4ZwD7AD9oYtwCuXd+5R5ye5CGt7ditsfX19UTgW1V1WTv/WG3LGALgrkAB95/CeX8nyeuB1wMs2WbFdA6VJKkLGwxCAFV1N8M6lzOSrAEOBc4FflhVT9rAoZmoTZJtpji2dZLRiNumcPzJwPuBpwPbTdJvjftz/DiOr6q/msI5R+3PMM5PAkcDb15fX0leuJ5zvxM4vape0oLdGdMZQFUdBxwHsNkOu07UvyRJXZtssfRubTZizF7AFcBaYEVbTE2S+yd5zLjDJ2xTVTcDVyV5cdu+WZItgVuArUeO/xbtFlSSRzLcqlo7jdo+DhxdVWvGbR/t9+kMa6BuHrf9QGDb1v7rwMFJHtz2LZ/q3wKrqtuBNwKHJFm+gb6+B+yX5OFj21sXy7hnofdhUy9dkiRNxWRrhLZiuDXzoySrgd2Bo6rqN8DBwHuTXAhcwLDW53cmafNa4A2tzzOB3wNWA3e1xcJvAj4MLGmzUCcCh40t2p6Kqrqqqj4wwa6jgFXt3McwzHDBsHZo3yTnAc8Gftr6+RHDra1T2zGnMSwin+o4fsawWPvw9fVVVdcx3ML6QnuvTmyHvw94T5LvMtxalCRJsyhV3jHpwWY77Fo7HPr38z0MSVKnLj/meZM3mkVJzq2qVZO181+WliRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLUraXzPQDNjcfuuIxzjnnefA9DkqQFxRkhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktQtg5AkSeqWQUiSJHXLICRJkrplEJIkSd0yCEmSpG4ZhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbhmEJElStwxCkiSpWwYhSZLULYOQJEnqlkFIkiR1yyAkSZK6ZRCSJEndMghJkqRuGYQkSVK3DEKSJKlbBiFJktStVNV8j0FzIMktwNr5Hscc2R64fr4HMQd6qROsdbGy1sVnIdW5S1WtmKzR0rkYiRaEtVW1ar4HMReSnNNDrb3UCda6WFnr4rMp1umtMUmS1C2DkCRJ6pZBqB/HzfcA5lAvtfZSJ1jrYmWti88mV6eLpSVJUrecEZIkSd0yCG3ikhyQZG2SS5K8ZYL9myU5se0/O8nKkX1/1bavTfKcuRz3xtjYWpOsTHJ7kgva4yNzPfbpmkKt+yY5L8ldSQ4et+/QJP/WHofO3ag3zgxrvXvkup48d6PeOFOo9c1JfpRkdZKvJ9llZN8mc11nWOdiu6Z/nmRNq+c7SXYf2bfYPoMnrHXBfwZXlY9N9AEsAX4CPAJ4AHAhsPu4Nn8BfKQ9fwVwYnu+e2u/GfDw1s+S+a7pPqp1JfCv813DLNe6EtgT+BRw8Mj25cCl7c9t2/Nt57um+6LWtu/W+a5hlmvdH9iyPf/PIz/Dm8x1nUmdi/SabjPy/IXAV9vzxfgZvL5aF/RnsDNCm7YnAJdU1aVV9RvgBOBF49q8CDi+Pf888IwkadtPqKo7quoy4JLW30I1k1o3NZPWWlWXV9Vq4Lfjjn0OcFpV3VBVvwJOAw6Yi0FvpJnUuqmZSq2nV9Wv28uzgIe155vSdZ1JnZuaqdR688jLBwJjC3MX3WfwBmpd0AxCm7YdgStHXl/Vtk3YpqruAm4CtpvisQvJTGoFeHiS85N8M8nT7uvBztBMrs1ivK4bsnmSc5KcleTFszu0WTfdWl8HnLKRx86nmdQJi/CaJjk8yU+A9wFvmM6xC8hMaoUF/Bnsvyy9aZtotmN8Al9fm6kcu5DMpNafATtX1S+T7AOclOQx4357WUhmcm0W43XdkJ2r6pokjwC+kWRNVf1klsY226Zca5LXAKuA/aZ77AIwkzphEV7TqvoQ8KEkrwLeBhw61WMXkJnUuqA/g50R2rRdBew08vphwDXra5NkKbAMuGGKxy4kG11rm3r+JUBVnctwn/uR9/mIN95Mrs1ivK7rVVXXtD8vBc4A9p7Nwc2yKdWa5JnAEcALq+qO6Ry7QMykzkV5TUecAIzNcm1K1xRmUOuC/wye70VKPjb+wTCjdynDQruxxWuPGdfmcO69gPiz7fljuPdCvUtZ2Av1ZlLrirHaGBb6XQ0sn++aZlLrSNtPsu5i6csYFtRu254v1lq3BTZrz7cH/o1xizcX0mOKP8N7M3xJ7Dpu+yZzXWdY52K8pruOPH8BcE57vhg/g9dX64L+DJ73AfiY4QWE5wIXtw+VI9q2oxl+ywLYHPgcw0K87wOPGDn2iHbcWuDA+a7lvqoVOAj4YfsP9zzgBfNdyyzU+ocMv6HdBvwS+OHIsf+hvQeXAH8637XcV7UCTwbWtOu6BnjdfNcyC7V+DfgFcEF7nLwpXteNrXORXtMPtM+fC4DTGQkPi/AzeMJaF/pnsP+ytCRJ6pZrhCRJUrcMQpIkqVsGIUmS1C2DkCRJ6pZBSJIkdcsgJEmSumUQkiRJ3TIISZKkbv1/UJI5v3S59rAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "metrics[['Full Model Recall','Selector Model Recall']].mean().sort_values().plot(kind='barh', ax=ax)\n",
    "ax.set_title('Mean Test Recall for RF Models');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlIAAAE/CAYAAACNXS1qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHVVJREFUeJzt3XmUZVV9t/HnazeTYZTBoAxtEnBACQgxjsigxojBAaI0iJBlNGoG3iQuY0RjaxKDiZHEqBHicsAYQHAIEk0QkDGBCAKNoBAREDQqCDRIcAB/7x9nF1yKqq5bu7urqpvns1atvvecfc/57XNu1/323ruqU1VIkiRp9h423wVIkiStrQxSkiRJnQxSkiRJnQxSkiRJnQxSkiRJnQxSkiRJnQxSkvQQlGTjJF9IckeSj893PXMhyYlJ3jJm2+8meeaarklrP4OUJJJcn+QnSbaatP2yJJVkyRzWcmiSH7avu5P8bOT5D1fhuI9Lcs+YbV/b+n1A7/nWAkuBjYEtquqwVT1YkueP3Ks7k3wtyaEj+zds1/Sukfv53WmONXH93zlp+8Ft+wdXtV5pdTFISZpwHcOHKwBJngRsNNdFVNUnqmrjqtoY+HXgOxPP27a5cDhwa/tzTiVZNEen2hG4uqrune0LkyyeZtc32z3aFPhT4KNJHjOpzWNH7ufPr+Q03wAOTTL6OfVK4JrZ1iutSQYpSRM+zvBBNeFw4PjRBkk2SPLuJN9K8r0kH0yyUdu3RZLTktyc5Lb2eLuR156d5M+TXNBGLE6fPAI2riTbJ/nXJLck+WaS147se0aSS9uU1XeT/FXbdS6waGQ0ZPdpjr0z8KvA7wAvTPKISft/M8nydvz/SbJf275VkuPbOW9LclLb/tokZ4y8fmJkZrv2/MQk723X4y7gaUlekuTydo4bkrx5Ug17J7kwyYp2Lw5J8qwkN44Gjza6d+EUfXwX8Ebg8HYtDk2yKMnbR+7th5Ns0to/Lsk9SV6d5Ebg8yu7PzX4LHA38MSVtV2JGxjC/T6thkcCuwFfmNSXA5NcleT2JGck2Wlk31PadbwzyT8D60967Uvavbw9yXlJnjBVISt5T0kGKUn3uRDYNMnj26jIy4F/ntTmXcDODB9ovwQ8Gviztu9hwEcYRjp2YPgQfd+k1x8C/BawDcOH2htmW2Sr7fPAfwKPAp4PvDnJs1uT9wHvrKpNgZ2Az7btewH3joyGXDrNKQ4Hzq+qUxg+zA8eOfezgOOAI4HNgf2AG9vuk4AAjwMeCbx/Ft16BfBWYBPgy8AdDNdqc+AlwBuSPL/V8EvAacDfAFsCewBXVtV5wE+AZ0867oPWP1XVnwDvAT7WrsUnGILjy4BnMVy3bVqbCYsYAuZjgRetrDNJHpbkN1t/rh37KjzY8dwf7g8FTgbum55N8kTgo8DrW73nAKcmWZxkQ4Z7fyzwCIYAdsDIa58KfIDh/bglw3X67DSjbdO9pySDlKQHmBiVei7wdeDbEzuSBHg18IdVdWtV3Qm8kxY0quoHVfWpqvq/tu8veeCHOsBHquqaqrob+CRDIJutZwIbVtW7quonVXUNQ4CbCDw/BXZOsmVV3VlVF4174DaacxjwL23TCTxweu+3gQ9W1Zeq6mdV9a2quqZNXz0LeH1V3d7qOncWfTqlqi5qx/xxVZ1ZVVe2519huFYT1/Iw4HPtWt9TVTdX1eVt3/EM4WliBOfZDAFvHIcCf1NVN1TVHcBRDFNrGWnzZ+3+3j3NMR6T5HaGEH0Cw/W4alKbK9sI0O1J/nqGmk4Gnp/k5xjel8dP2r8U+ExVnV1VP2F4P24F7MkQnH9cVR+oqp+2sLh85LW/A7yvqi6pqnur6jhgA4ZgOln3e0rrPoOUpFEfZxgJOYIHf2htDTwcuGTigxD497adJA9PcmybirqDYSpt8zxwzc/o4uL/Y1jsPFs7AktGPoxvB/4ImFhvcziwK3BNkouS/Nosjr1PO87J7fkngKckeVx7vj1Tj7BsD3y/BcgeN44+aVNJ52SYJl3BcD8mpkGnqwGGe/bSNhqzFPhiVd0yZg2PYhiBm3ADwxq5ianNn1XVd2Y4xnVVtTmwGcPI3b5TtNmlqjZvX29c2cHa9TwTeBuwXlVdsrKa23qvbzOMlD4KuGlS+9H+7cgwkjn6Ptq6vXayVXlPaR1nkJJ0n6qaWJfyAuDTk3bfwjDSMPpBuNnIAvA/Zpj2+dU2BbJX2x5WrxuBr4/UsHlVbVJVL2l9+FpVvZxhque9wKeTrA/UGMc+nOH74pUZfqLs3Pa6iemlG4FfnKambZJMFQzvYgigE6ZaYD25tk8yjCRtX1WbMUxfTVzH6Wqgqq5jGHX5DYaRq9n8WoPvMISLCRPTs7dOU+O0qupHDOH2aRNTkqvgeIYp4MnBHibV3EL7oxnC1P8C201qv8PI4xsZRthG30cPr6rJ7/uVvackg5SkB3kVsG9V3TW6sap+BvwTcEySbQCSPHrkX+ebMHzw3p5hgfbb1lB957dz/7+2cHtxkl2TPLltf2WbgrkXWMEQAH4GfJ9hsfkOUx20haCXMoz+7Dby9QbgsDbt9yHgd5Ls1dYBbZ9k5xZgzgXel2SzJOsnmQiSlwG7J9klycO5f03ZlNpU2sbAD6rqR0meDvzmSJPjGRbBv6QtEN86ya6T9r8V+AXgczNezfudwLAWa4e2yPwvgH+pqrED1KgWpv6OVX8ffBF4HjDVrzw4CXhJux/rAW8CfgBczHA/Nsyw2H9xkqUMo0oTjgN+P8meGWyc5IB2jx5gJe8pySAl6YGq6tqqunia3X/C8GPpF7bpuzMYRqFg+NDciGHk6kKGab81Ud9PGUbMns4wVXMz8I/cP034QuDqJHcCfwW8rK0lug34a+6fmpy8PusghtGXE6rquxNfDB+4mzKEy/OA1zIsUl7BMO00MeqxFFgP+B+GKczXtXqvaOc9j2Hd2dkz9K/aOd7d+vBG7p9qpKquZVjs/WbgNobQsMvIIU5m+EGAT1bVj1d2rkn+kWEU8j8Zpg5vZRhVWhXHAY9P8tzeA7T1S2dU1Yop9i1nCP7HMrwP9gNe1O733QwL9V/PcJ32ZyRYVtUFwB+0197O8GsVDmHqkbcp31O9fdK6JZ3/2JAkLUBt5OxbwMFVdf581yOt6xyRkqR1y1LgDkOUNDem++20kqS1TIZfvrmEYYpK0hxwak+SJKmTU3uSJEmdDFKSJEmdXCOlGW211Va1ZMmS+S5DkqQ5c8kll9xSVVvP1M4gpRktWbKEiy+e7tcKSZK07klyw8ytnNqTJEnqZpCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqZJCSJEnqtHi+C9Ba4DuXwrLN5rsKSZIebNmKeT29I1KSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmdDFKSJEmd5jVIJbk3yWUjX0tmaH99kq3a4x9O06aSfHzk+eIkNyc5bZa13XeunjZJdm+1/NpszitJktYei+f5/HdX1W6r+Zh3AU9MslFV3Q08F/j2aj7HOJYC57c//2NNnSTJ4qq6Z00dX5IkTW/BTe0lOSLJ+0aen5Zk71ke5gvA/u3xUuCEkeM9IslnkyxPcmGSXdv2LZOcnuTSJMcCGXnNK5L8dxs1OzbJohn6EOAg4AjgeUk2HNn3ynbuyydGzpI8Msln2rbLkzw9yZIkXx153RuSLGuPz07yziTnAEcm+Y0kF7Xaz0jyyNZu4yQfSXJFO+eBSV6V5JiR4746yXtmeX0lSRLzH6Q2GpnW+8xqPO6JwMEtwOwKXDSy7+3ApVW1K/Bm4Pi2/W3A+VW1O3AqsANAkscDLwee0UbP7gUOneH8zwCuq6prgbOBF7Rj7QIcBexbVb8MHNnavxc4p217MnDlGH3cvKqeXVV/yzDy9dRW+4nAG1ubtwIrqupJrb9ntf0HJFmvtfkt4CNjnE+SJE2yLk7tUVXL23qrpcDnJ+1+JnBga3dWG4naDNgLeGnb/m9Jbmvt9wP2AL48DDSxEfD9GUpYyhBYaH8eBnwa2Bc4papuaee5tbXZF3hl23YvsCLJFjOc46SRx9sBJyXZFlgfuK5tfw5w8ESjqroNIMlZwAuTfA1Yr6qumHzwJK8BXgOww2aZvFuSJDH/QWoq9/DAkbINp2s4g1OBdwN7A1uObJ8qFdSkP0cF+FhV/ek4J23TfgcyjPoc1V6/ZZJN2uOpzjGVma7DXSOP/wF4T1Wd2qZBl43UPtX5PsQwGvd1phmNqqrjgOMA9nzUonFrliTpIWW+p/amcj2wW5KHJdkeeErncT4MvGOK0ZZzaVNzLXTcUlV3TNr+68DEiNCZwEFJtmn7HpFkx5Wc9znA5VW1fVUtqaodgU8BL27HelmSLSeONXKO17Vti5JsCnwP2KaNmG0AvHAl59yM+xfUHz6y/XTg9yaeTIxyVdVFwPbAIYysH5MkSbOzEIPUBQxTU1cwjCh9pecgVXVTVf39FLuWAXsmWQ4czf3B4+3AXkm+AjwP+FY7zlXAW4DT22u+CGy7klMvBSav9/oUcEhVXQn8JXBOksuBiUXeRwL7JLkCuATYpap+CryDYX3XaQyjR9NZBpyc5DzglpHtfwFskeSr7Xz7jOz7JHDBxHSfJEmavVQ5a/NQ1H6v1jFVdeZMbfd81KK6+DUbz0FVkiTN0rIVa+SwSS6pqj1narcQR6S0BiXZPMk1DAv9ZwxRkiRpegtxsbnWoKq6Hdh5vuuQJGld4IiUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSJ4OUJElSp8XzXYDWAo/aHZZdPN9VSJK04DgiJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1MkgJUmS1GnxfBeghe+Kb69gyZv+bb7LkCTpAa4/ev/5LsERKUmSpF4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4GKUmSpE4zBqkkRyW5MsnyJJcl+dUZ2n80yUGzLSTJkiSHzPZ1Uxynknx85PniJDcnOW2Wx7k+yVa9bZLs3mr5tdmcV5IkrT1WGqSSPA14IfDkqtoVeA5w4xqqZQkwqyCVZPEUm+8Cnphko/b8ucC3V620LkuB89ufa8w010CSJM2BmUaktgVuqaofA1TVLVX1HYAkeyQ5J8klSf4jybaTXzxdmyS/lOSMJJcn+UqSXwSOBp7VRr3+MMmGST6S5IoklybZp732iCQnJ/kccPo0dX8B2L89XgqcMFLTI5J8to2wXZhk17Z9yySnt3MdC2TkNa9I8t+ttmOTLFrZRUsS4CDgCOB5STYc2ffKdu7LJ0bOkjwyyWfatsuTPL2N0H115HVvSLKsPT47yTuTnAMcmeQ3klzUaj8jySNbu41HruHyJAcmeVWSY0aO++ok71lZfyRJ0tRmClKnA9snuSbJB5I8GyDJesA/AAdV1R7Ah4G/HH3hDG0+Aby/qn4ZeDrwv8CbgPOqareqOgb4XYCqehJDGPrYSCB5GnB4Ve07Td0nAge39rsCF43seztwaRthezNwfNv+NuD8qtodOBXYofXj8cDLgWdU1W7AvcChM1y3ZwDXVdW1wNnAC9qxdgGOAvZtfT+ytX8vcE7b9mTgyhmOD7B5VT27qv6WYeTrqa32E4E3tjZvBVZU1ZNaf89q+w9o9wfgt4CPjHE+SZI0yUqnharqh0n2AJ4F7AOclORNwMXAE4EvDoMvLGIIQ6MeO1WbJJsAj66qz7Rz/AigtRn1TIYgRlV9PckNwM5t3xer6taV1L08yRKGAPb5KY57YGt3VhuJ2gzYC3hp2/5vSW5r7fcD9gC+3GrcCPj+dOduljIEFtqfhwGfBvYFTqmqW9p5JvqwL/DKtu1eYEWSLWY4x0kjj7djuDfbAusD17XtzwEOnmhUVbcBJDkLeGGSrwHrVdUVkw+e5DXAawAWbbr1DKVIkvTQNOP6mvbBfjZwdpIrgMOBS4Arq+ppK3lppmqTZNMxa3tQshpx1xivPxV4N7A3sOUMx61Jf06u42NV9adjnJM27Xcgw6jPUe31W7YAmWnOMZV7eOCI4YaT9o9eg38A3lNVpybZG1g2UvtU5/sQw2jc15lmNKqqjgOOA9hg253GrVmSpIeUmRabPzbJTiObdgNuAK4Gtm6L0UmyXpu2GjVlm6q6A7gpyYvb9g2SPBy4E9hk5PXn0qbQkuzMMNV29Sz69mHgHVOMtowed2+GNWB3TNr+68DEiNCZwEFJtmn7HpFkx5Wc9znA5VW1fVUtqaodgU8BL27HelmSLSeONXKO17Vti1rY/B6wTRsx24Bh0f90NuP+BfWHj2w/Hfi9iScTo1xVdRGwPcPi/hOQJEldZlojtTHD2qSrkiwHngAsq6qfMCymfleSy4HLGNY63WeGNocBf9CO+Z/AzwPLgXvaYus/BD4ALGqjYCcBR0wseh9HVd1UVX8/xa5lwJ7t3Edzf/B4O7BXkq8AzwO+1Y5zFfAW4PT2mi8yLMKfzlLgM5O2fQo4pKquZFgndk67JhOLvI8E9ml9vQTYpap+CryDYX3XaQyjR9NZBpyc5DzglpHtfwFskeSr7Xz7jOz7JHDBxHSfJEmavVQ5a/NQlOH3ah1TVWfO1HaDbXeqbQ//uzmoSpKk8V1/9P4zN+qU5JKq2nOmdv5m84eYJJsnuQa4e5wQJUmSpucvc3yIqarbuf+nHyVJ0ipwREqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKmTQUqSJKnT4vkuQAvfkx69GRcfvf98lyFJ0oLjiJQkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVKnVNV816AFLsmdwNXzXcdqthVwy3wXsRrZn4VvXeuT/Vn41rU+zXV/dqyqrWdqtHguKtFa7+qq2nO+i1idkly8LvXJ/ix861qf7M/Ct671aaH2x6k9SZKkTgYpSZKkTgYpjeO4+S5gDVjX+mR/Fr51rU/2Z+Fb1/q0IPvjYnNJkqROjkhJkiR1MkjpPkmen+TqJN9I8qYp9m+Q5KS2/6IkS+a+yvGN0Z+9knwlyT1JDpqPGmdrjD79UZKrkixPcmaSHeejznGN0Z/XJrkiyWVJzk/yhPmoc1wz9Wek3UFJKsmC+wmkyca4R0ckubndo8uS/PZ81Dmuce5Rkpe1v0dXJvmXua5xNsa4P8eM3Jtrktw+H3XOxhh92iHJl5Jc2r7XvWA+6rxPVfnlF8Ai4FrgF4D1gcuBJ0xq83rgg+3xwcBJ8133KvZnCbArcDxw0HzXvJr6tA/w8Pb4devAPdp05PEBwL/Pd92r0p/WbhPgXOBCYM/5rns13KMjgPfNd62rsT87AZcCW7Tn28x33avSn0ntfx/48HzXvRru0XHA69rjJwDXz2fNjkhpwlOAb1TVN6vqJ8CJwIsmtXkR8LH2+BRgvySZwxpnY8b+VNX1VbUc+Nl8FNhhnD59qar+rz29ENhujmucjXH6c8fI058DFvKiznH+DgH8OfDXwI/msrhO4/ZpbTFOf14NvL+qbgOoqu/PcY2zMdv7sxQ4YU4q6zdOnwrYtD3eDPjOHNb3IAYpTXg0cOPI85vatinbVNU9wApgyzmpbvbG6c/aZrZ9ehXwhTVa0aoZqz9JfjfJtQzh4w/mqLYeM/Ynye7A9lV12lwWtgrGfc8d2KZYTkmy/dyU1mWc/uwM7JzkgiQXJnn+nFU3e2N/T2jT/I8BzpqDulbFOH1aBrwiyU3A5xlG2uaNQUoTphpZmvyv/3HaLBRrU63jGrtPSV4B7An8zRqtaNWM1Z+qen9V/SLwJ8Bb1nhV/VbanyQPA44B/njOKlp149yjzwFLqmpX4AzuH7VeiMbpz2KG6b29GUZwPpRk8zVcV6/ZfJ87GDilqu5dg/WsDuP0aSnw0araDngB8PH292teGKQ04SZg9F+S2/Hg4dL72iRZzDCkeuucVDd74/RnbTNWn5I8BzgKOKCqfjxHtfWY7T06EXjxGq1o1czUn02AJwJnJ7keeCpw6gJfcD7jPaqqH4y8z/4J2GOOausx7ve5f62qn1bVdQz/z+hOc1TfbM3m79DBLPxpPRivT68CPglQVf8FbMjw//DNC4OUJnwZ2CnJY5Ksz/CX7tRJbU4FDm+PDwLOqrbabwEapz9rmxn71KaOjmUIUQt5bQeM15/RD7D9gf+Zw/pma6X9qaoVVbVVVS2pqiUMa9gOqKqL56fcsYxzj7YdeXoA8LU5rG+2xvm+8FmGH9ogyVYMU33fnNMqxzfW97kkjwW2AP5rjuvrMU6fvgXsB5Dk8QxB6uY5rXLUfK/Q92vhfDEMkV7D8BMTR7Vt72D4Zg/Dm/Vk4BvAfwO/MN81r2J/foXhXz93AT8ArpzvmldDn84Avgdc1r5One+aV7E/fw9c2fryJWCX+a55Vfozqe3ZLPCf2hvzHv1Vu0eXt3v0uPmueRX7E+A9wFXAFcDB813zqr7nGNYUHT3fta7Ge/QE4IL2nrsMeN581utvNpckSerk1J4kSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVIng5QkSVKn/w84mzLxUMtVxQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "metrics[['Full Model Accuracy','Selector Model Accuracy']].mean().sort_values().plot(kind='barh', ax=ax)\n",
    "ax.set_title('Mean Test Accuracy for RF Models');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
